<?php
class Crawler{
	var $url = null;
	var $limit = 15;

	public function __construct($url, $limit = 15){
		$this->url = $url;
		if(intval($limit) > 0) $this->limit = $limit;
	}
	public function curl($url){
		try{
			// create a new cURL resource
			$ch = curl_init();

			curl_setopt($ch, CURLOPT_URL, $url);
			curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);
			curl_setopt($ch, CURLOPT_USERAGENT, ' Mozilla/5.0 (Windows NT 6.3; WOW64; rv:27.0) Gecko/20100101 Firefox/27.0');
			curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, false);// require to get data from https
			curl_setopt($ch, CURLOPT_FOLLOWLOCATION, true);
			curl_setopt($ch, CURLOPT_TIMEOUT, 900);
			curl_setopt($ch, CURLOPT_AUTOREFERER, true);

			$data = curl_exec($ch);
			curl_close($ch);
			return $data;
		}
		catch(Exception $e){
			echo $e->getMessage();
		}
		return null;
	}
	public function getLinks($filterType, $strFilter, $arrDetail){
		$data = $this->curl($this->url);
		$doc = phpQuery::newDocumentHTML($data);
		$arrLink = array();
		$arrContent = array();
		$demoDetail = '';
		if(!isset($arrDetail['removeDom'])) $arrDetail['removeDom'] = null;
		if($filterType == 'RSS'){
			$urls = $this->rssExtractor();
			foreach($urls as $feed){
				$links[] = $feed['link'];
				$title[] = $feed['title'];
			}
		}
		else{
			$linkDom = 'a';
			if($filterType == 'DOM'){
				$linkDom = $strFilter;
			}
			$links = pq($linkDom, $doc);
		}

		if(count($links) > 0){
			$i = 0;
			$href = '';
			$linkTitle = '';
			foreach($links as $link){
				if($i < $this->limit){
					if($filterType == 'RSS'){
						$href = $link;
						$linkTitle = $title[$i];
					}
					else{
						$href = pq($link)->attr('href');
						$linkTitle = pq($link)->html();
					}
					$linkTitle = strip_tags($linkTitle);
					if($arrDetail['urlAdd'] != '') $href = $arrDetail['urlAdd'] . $href;
					$href = preg_replace('/(\w+)\/\//', '$1/', $href);
					if(in_array($href, $arrLink)) continue;

					if($filterType == 'HASWORD'){
						if(strstr($href, $strFilter) && !in_array($href, $arrLink)){
							$arrLink[] = $href;
							if(!$arrDetail['demo']){
								// get a demo detail page
								$detail = $this->getDetail($href, $arrDetail['detailType'], $arrDetail['detailString'], $arrDetail['removeDom']);

								$arrContent[$i]['body'] = $detail['content'];
								// $arrContent[$i]['title'] = $detail['title'];
								$arrContent[$i]['title'] = $linkTitle != '' ? $linkTitle : $detail['title'];
							}
							$i++;
						}
					}
					elseif($filterType == 'REGX'){
						if(preg_match('/'.$strFilter.'/is', $href) && !in_array($href, $arrLink)){
							$arrLink[] = $href;
							if(!$arrDetail['demo']){
								// get a demo detail page
								$detail = $this->getDetail($href, $arrDetail['detailType'], $arrDetail['detailString'], $arrDetail['removeDom']);

								$arrContent[$i]['body'] = $detail['content'];
								// $arrContent[$i]['title'] = $detail['title'];
								$arrContent[$i]['title'] = $linkTitle != '' ? $linkTitle : $detail['title'];
							}
							$i++;
						}
					}
					elseif($filterType == 'RSS'){
						// $urls = $this->rssExtractor();
						// $urls = $links;
						if(!empty($links)){
							$arrLink[] = $link;

							if(!$arrDetail['demo']){
								// get a demo detail page
								$detail = $this->getDetail($href, $arrDetail['detailType'], $arrDetail['detailString'], $arrDetail['removeDom']);

								$arrContent[$i]['body'] = $detail['content'];
								// $arrContent[$i]['title'] = $detail['title'];
								$arrContent[$i]['title'] = $linkTitle != '' ? $linkTitle : $detail['title'];
							}
							$i++;
						}
					}
					else{
						$arrLink[] = $href;
						if(!$arrDetail['demo']){
							// get a demo detail page
							$detail = $this->getDetail($href, $arrDetail['detailType'], $arrDetail['detailString'], $arrDetail['removeDom']);

							$arrContent[$i]['body'] = $detail['content'];
							// $arrContent[$i]['title'] = $detail['title'];
							$arrContent[$i]['title'] = $linkTitle != '' ? $linkTitle : $detail['title'];
						}
						$i++;
					}
				}
			}
			if($arrDetail['demo']){
				// get a demo detail page
				$detail = $this->getDetail($href, $arrDetail['detailType'], $arrDetail['detailString'], $arrDetail['removeDom']);
				$demoDetail = $detail['content'];
			}
		}
		$demoDetail = $demoDetail != '' ? $demoDetail : $arrContent;
		return array('links' => $arrLink, 'detail' => $demoDetail);
	}
	public function getDetail($detailUrl, $filterType, $strFilter, $removeDom = null){
		$masterContent = $this->curl($detailUrl);
		$doc = phpQuery::newDocumentHTML($masterContent);
		$pattern = '#<title.*?>(.*?)</title>#is';
		if(preg_match($pattern, $masterContent, $match)){
			$title = $match[1];
		}
		else{
			$title = pq('title', $doc)->text();
		}
		$content = '';
		if($filterType == 'DOM'){
			$domContent = pq($strFilter, $doc);
			// remove DOM
			if($removeDom != null){
				$arrRemove = explode(',', $removeDom);
				foreach($arrRemove as $remove){
					pq(trim($remove), $domContent)->remove();
				}
			}
		}
		elseif(preg_match('/'.$strFilter.'/is', $detailUrl, $match)){
			$content = $match[1];
		}
		// remove script tags
		$content = $domContent->html();
		$content = preg_replace('#<script(.*?)>(.*?)</script>#is', '', $content);
		
		return array('content' => $content, 'title' => $title);
	}
	public function rssExtractor(){
		$gapi = 'https://ajax.googleapis.com/ajax/services/feed/load?v=2.0&output=json&q=';
		try{
			// create a new cURL resource
			$ch = curl_init();

			curl_setopt($ch, CURLOPT_URL, $gapi . urlencode($this->url) . '&num=' . $this->limit);
			curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);
			curl_setopt($ch, CURLOPT_USERAGENT, ' Mozilla/5.0 (Windows NT 6.3; WOW64; rv:27.0) Gecko/20100101 Firefox/27.0');
			curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, false);// require to get data from https

			$data = curl_exec($ch);
			curl_close($ch);
			$objData = json_decode($data, true);
			if(!empty($objData['responseData']['feed']['entries'])){
				return $objData['responseData']['feed']['entries'];
			}
		}
		catch(Exception $e){
			echo $e->getMessage();
		}
		return null;
	}
	public function getUrlData($url, $params = null, $mRequest = false, $binaryTransfer = false){
	    $userAgent = 'Mozilla/5.0 (Windows; U; Windows NT 5.1; ru; rv:1.8.0.9) Gecko/20061206 Firefox/1.5.0.9';
	    $header = array(
	    "Accept: text/xml,application/xml,application/xhtml+xml,text/html;q=0.9,text/plain;q=0.8,image/png,*/*;q=0.5",
	    "Accept-Language: ru-ru,ru;q=0.7,en-us;q=0.5,en;q=0.3",
	    "Accept-Charset: windows-1251,utf-8;q=0.7,*;q=0.7",
	    "Keep-Alive: 300");
	    if($mRequest){
	        $userAgent = 'Mozilla/5.0 (iPhone; U; CPU iPhone OS 4_3_2 like Mac OS X; en-us) AppleWebKit/533.17.9 (KHTML, like Gecko) Version/5.0.2 Mobile/8H7 Safari/6533.18.5';
	    }
	    $ch = curl_init();
	    curl_setopt($ch, CURLOPT_HTTPHEADER, $header);
	    curl_setopt($ch, CURLOPT_URL, $url);
	    curl_setopt($ch, CURLOPT_SSL_VERIFYHOST, 0);
	    curl_setopt($ch, CURLOPT_USERAGENT, $userAgent);
	    curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);
	    curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, 0);
	    curl_setopt($ch, CURLOPT_FOLLOWLOCATION, 1);
	    if($binaryTransfer){
	        curl_setopt($ch, CURLOPT_BINARYTRANSFER, true);
	    }
	    if ($params) {
		    curl_setopt($ch, CURLOPT_CUSTOMREQUEST, "POST");
		    curl_setopt($ch, CURLOPT_POST, 1);
		    curl_setopt($ch, CURLOPT_POSTFIELDS, http_build_query($params));
	    }
	    $result = curl_exec($ch);
	    if ($result) {
	    	return $result;
	    } else {
	    	return curl_error($ch);
	    }
	    curl_close($ch);
	}
	public function removeLinks($content){
		$pattern = '/<a.*?href=.*?>(.*?)<\/a>/is';
		$content = preg_replace($pattern, '<strong class="lnk2bold">$1</strong>', $content);
		return $content;
	}
}